#!/usr/local/python/bin/python
# coding=utf-8
__author__ = 'cz'

# 文件加载
# Example:
# python2.7 load.py biz_code biz_date
# Return:
# 0 成功
# 1 出错

import sys
from datetime import datetime

from model.FileMetaInfo import *
from tdtools import Common_tdsql_prd as Common
from tdtools import Config_tdsql_prd as Config
from tdtools import LogUtil
from tdtools.FileUtil import FileUtil
from tdtools.ShellUtil import ShellUtil
from tdtools.TDSQLCon import *
from tdtools.DateUtils import DateUtils

'''
Python 2.7默认使用ASCII编码，当代码中涉及非ASCII字符（如中文字符）时，
若未明确指定编码方式，会尝试用ASCII解码，导致UnicodeDecodeError。
'''
default_encoding = 'utf-8'
if sys.getdefaultencoding() != default_encoding:
    reload(sys)
    sys.setdefaultencoding(default_encoding)

shellCode = 'LOAD'
bizType = 'LOAD'

_logger=LogUtil._logger
_appInfoObj = Common._appInfoObj
_fileInfoObj = Common._fileInfoObj
_dbInfoObj = Common._dbInfoObj

_bizdate = ''
_dataFile = ''
_tranFile = ''
_hdfsDir = ''
_hdfsFile = ''
_dbCode = ''
_tbName=''
_retainDaysNum = 7
_retainMonthsNum = 13
_flgArrays = {}
_fileMetaInfo= FileMetaInfo()
_tDSQLCon=TDSQLCon()

# 初始化信息
def init():
    global _fileMetaInfo,_fileInfoObj, _appInfoObj, _dbInfoObj,_tDSQLCon, _bizdate, _dataFile, _tranFile, _hdfsDir, _hdfsFile, _retainDaysNum, _retainMonthsNum, _dbCode
    # 检查参数，初始化日志模块
    Common.checkArgs()

    Common.initFileInfo()
    _fileInfoObj = Common._fileInfoObj

    Common.initAppInfo()
    _appInfoObj = Common._appInfoObj

    Common.initDbInfo()
    _dbInfoObj = Common._dbInfoObj

    Common.initTdConnInfo()
    _tDSQLCon=Common._tDSQLCon


    # 业务日期
    _bizdate = Common.getBizDate()
    # 载入文件路径
    _dataFile = Common.getSourceFilePath()

    # 元数据文件路径
    metaFile = Common.getMetaFilePath()

    _fileMetaInfo=Common.getFileMetaInfo(metaFile)

    # 数据库编码
    _dbCode = _dbInfoObj.getDbCode()

    # 需要保留的天数
    _retainDaysNum = _fileInfoObj.getKeepDayNum()

    # 保留月末的月数
    _retainMonthsNum = _fileInfoObj.getKeepMonthNum()




# 获取真实文件路径_dataFile
def checkLines():
    global _dataFile,_fileMetaInfo
    dataLineNums=int(_fileMetaInfo.get_data_line_nums())
    flgLineNums = int(FileUtil.getFileLineNums(_dataFile))
    message = '标记文件中记录'+ str(dataLineNums) + '行，源文件实际'  + str(flgLineNums) + '行'
    _logger.info(message)
    if not dataLineNums == flgLineNums:
        _logger.error("标记文件中文件行数和源文件不匹配")
        Common.exit(Config.RUNNING_ERROR, shellCode, message)


# 获取flg内容
def getFlgFileContent():
    flgFile = Common.getDataFileDir() + Common.getDataFlgFileName()
    global _flgArrays
    if FileUtil.fileExists(flgFile):
        # 获取flg文件的最后一行信息
        lastNum = FileUtil.getFileLineNums(flgFile)
        lineStr = FileUtil.getLineContent(flgFile, lastNum)
        _flgArrays = lineStr.split(' ')
        return len(_flgArrays) > 2
    else:
        return False


# 获取文件大小
def getFlgSize():
    return int(_flgArrays[1])


# 获取文件行数
def getFlgLineNums():
    global _dataLines
    _dataLines = int(_flgArrays[2])
    return _dataLines

def countNum():
    global _dbInfoObj

# 拼接载数命令
def initLoadSql():
    global _dbInfoObj,_fileMetaInfo
    host = _dbInfoObj.getSqlHost()
    port = _dbInfoObj.getSqlPort()
    user = _dbInfoObj.getSqlUser()
    password = _dbInfoObj.getSqlPwd()
    db_name=_dbInfoObj.getDbName()

    # 内网版本：sql = "/mysql/app/bin/mysql
    sql = "mysql -c -h%s -P%s -u%s -p'%s' %s -e \"load data local infile '%s' ignore into table %s fields terminated by %s lines terminated by '\\n'\"" % (host,port,user,password,db_name,_fileMetaInfo.get_file_name(),_fileInfoObj.getTableName(),_fileInfoObj.getColSeprator())
    _logger.info("执行文本载数语句：%s"%sql)
    return sql

# 文本载数主流程
def process():
    global _dataFile,_tDSQLCon,_fileInfoObj,_fileMetaInfo
    if FileUtil.fileExists(_dataFile):
        conn = _tDSQLCon.get_connection()
        try:
            status = int(_fileInfoObj.getDataType())
            if status == 2:
                # 全量类型数据处理
                with conn.cursor() as cursor:
                    tb_tb_name = _fileInfoObj.getTableName()
                    tb_db_name = _tDSQLCon.db + "." + tb_tb_name
                    tb_db_list = tb_db_name.split(".")
                    partition_check_sql = "select count(*) from information_schema.partitions where TABLE_SCHEMA ='%s' and TABLE_NAME ='%s' and PARTITION_NAME is not null  " % (
                    tb_db_list[0], tb_db_list[1])
                    _logger.info("检查是否 %s 为分区表,执行sql:%s" % (tb_db_name, partition_check_sql))
                    cursor.execute(partition_check_sql)
                    par_res = cursor.fetchone()
                    par_num = par_res[0]
                    _logger.info("%s分区个数：%s" % (tb_db_name, par_num))
                    if par_num < 1:
                        truncate_sql = "truncate table %s " % _fileInfoObj.getTableName()
                    else:
                        truncate_sql = "delete from  %s " % _fileInfoObj.getTableName()
                    _logger.info("数据加载类型为【全量】，预先清理数据，执行sql： %s " % truncate_sql)
                    cursor.execute(truncate_sql)
                conn.commit()
            
            # 增量类型数据处理
            # 增量类型载数时，先清空该业务日期如20250409对应增量数据，再进行载数
            # DT数据如果为2025-04-09 00:00:00，则是增量数据需要清理
            # DT数据如果为空白则说明是增量处理有问题需要清理
            with conn.cursor() as cursor:
                delete_day = DateUtils.getLongDate(_bizdate)
                date_obj = datetime.strptime(delete_day,"%Y-%m-%d")
                datetime_obj = date_obj.strftime("%Y-%m-%d %H:%M:%S")
                delete_sql = "delete from %s  where DT = %%s or DT IS NULL" % _fileInfoObj.getTableName()
                _logger.info("【增量】情况 %s 清空增量数据,执行sql：%s" % (_fileInfoObj.getTableName(), delete_sql))
                cursor.execute(delete_sql,datetime_obj)
                
                sql = "select count(*) from %s" % _fileInfoObj.getTableName()
                cursor.execute(sql)
                res_bef = cursor.fetchone()
                _logger.info("清理增量数据后，表中条数为：【%s】" % res_bef)
            conn.commit()

            # 执行载数语句
            shell = initLoadSql()
            if not ShellUtil.execShell(shell):
                _logger.error("执行载数语句：%s【失败】" % shell)
                Common.exit(Config.RUNNING_ERROR, shellCode, "执行载数语句：%s【失败】" % shell)
            else:
                _logger.info("执行载数语句【成功】")

            with conn.cursor() as cursor:
                # 查询导入后的总条数
                sql = "select count(*) from %s" % _fileInfoObj.getTableName()
                cursor.execute(sql)
                res_aft = cursor.fetchone()
                _logger.info("导入数据后条数【%s】" % res_aft)

                # 更新增量字段DT(载数文本没有该字段，数据库显示为NULL)
                update_sql = "update %s set DT = %%s set DT %%s where DT IS NULL" % _fileInfoObj.getTableName()
                cursor.execute(update_sql,DateUtils.getLongDate(_bizdate))
                update_rows = cursor.rowcount
                _logger.info("更新增量字段DT条数为【%s】" % update_rows)
            conn.commit()

            # 本次载数导入数据条数
            load_num= res_aft[0] - res_bef[0]
            _logger.info("本次载数导入数据条数【%s】" % load_num)
            # 本次预期导入数据条数
            except_num=int(_fileMetaInfo.get_data_line_nums())
            _logger.info("本次预期导入数据条数【%s】" % except_num)

            if not load_num==except_num:
                _logger.error("本次载数导入数据条数【%s】行和本次预期导入数据条数【%s】行存在【差异】" % (load_num, except_num))
                Common.exit(Config.RUNNING_ERROR, shellCode, "本次载数导入数据条数 %s 行和本次预期导入数据条数 %s 行存在【差异】" % (load_num, except_num))
                # TODO Common.extExit(shellCode)
            else:
                _logger.info("本次载数导入数据条数和本次预期导入数据条数【一致】")

        except Exception as error:
            _logger.error("本次载数导入数据【异常】 %s " % error, exc_info=True)
        finally:
            _tDSQLCon.close_connection()
    else:
        _logger.error("DAT源文件【不存在】")
        Common.exit(Config.RUNNING_ERROR, shellCode, 'DAT源文件【不存在】')


# 执行上传方法
def main():
        global _logFile,_logPrefix
        # 初始化信息,TDSQL的连接信息、元数据文件路径、源文件路径、分割符、库表名称
        init()

        #检测数据行数
        checkLines()

        #处理数据
        process()



def global_exception_handler(type, value, tb):
    global _logger
    # 记录异常信息
    _logger.log_exception("异常：", exc_info=(type, value, tb))



if __name__ == "__main__":
    # todo 全局异常处理器
    # sys.excepthook = global_exception_handler
    try:
        main()
    except Exception as e:
        _logger.error(e, exc_info=True)
        Common.exit(Config.RUNNING_ERROR, shellCode, e)
    else:
        _logger.info("执行完成")
        Common.exit(Config.RUNNING_SUCCESS, shellCode,"脚本执行成功")